'''
Created on 2014. 1. 3.

@author: jhpark
'''
from scrapy.spider import BaseSpider
from scrapy.selector import Selector

from crawler.items import CrawlerItem

class DMozSpider(BaseSpider):
    '''
    classdocs
    '''
    name = 'dmoz'
    allowed_domains = ["dmoz.org"]
    start_urls = [
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Books/",
        "http://www.dmoz.org/Computers/Programming/Languages/Python/Resources/"
        ]
   
    def parse(self, response):
        sel = Selector(response)
        sites = sel.xpath('//ul/li')
        items = []
        for site in sites:
            item = CrawlerItem()
            item['title'] = site.xpath('a/text()').extract()
            item['link'] = site.xpath('a/@href').extract()
            item['desc'] = site.xpath('text()').extract()
            items.append(item)
        
        return items
            
        
        